package com.utils

import org.apache.spark.{SparkConf, SparkContext}

/**
  * ClassName:DirectToRedis
  * Package:com.beans
  * Desciption:将字典数据放到redis
  *
  * @date:2019 /8/24 0:35
  * @author:17611219021 @sina.cn
  */
object DirectToRedis {
  def main(args: Array[String]): Unit = {
    if(args.length !=1){
      println(
        """
          |com.dmp.fileToParquet
          |参数：
          | inputpath
        """.stripMargin)
      sys.exit()
    }
    val conf: SparkConf = new SparkConf()
    conf.setAppName(s"${this.getClass.getSimpleName}")
    conf.setMaster("local[*]")
    //指定keyo序列化器与snappy格式，默认是gzip格式
    conf.set("spark.serializer","org.apache.spark.serializer.KryoSerializer")
    conf.set("spark.sql.parquet.compression.codec","snappy")
    val sc: SparkContext = new SparkContext(conf)
    val Array(inputpath)= args
    sc.textFile(inputpath).map(line=>{
      val arr=line.split(",",-1)
      (arr(0),arr(1))
    }).foreachPartition(itr=>{
      var jedis=JedisPools.getJedis()
      itr.foreach(it=>{
        jedis.set(it._1,it._2)
      })
    })
    sc.stop()
  }
}
